package invertedindex;

import java.io.*;
import java.sql.*;
import java.util.*;

public class Index {
    File[] files;
    SortedMap<String, SortedMap<Integer, Integer>> wordDocumentMap = new TreeMap<String, SortedMap<Integer,Integer>>();
    Set<String> stopWords = new HashSet<String>();
    
    Connection conn;
    PreparedStatement stmt;
    
    public Index(String dirPath, String stopwordsfile) throws SQLException {
    	
    	DriverManager.registerDriver (new oracle.jdbc.driver.OracleDriver());
        conn = DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:TEACH", "wunan","uviccsc");
     
        // Make a list with files.
        File dir = new File(dirPath);
        this.files = dir.listFiles();
        
        // Sort files by name.
        Arrays.sort(this.files, new Comparator<File>()
        {
            public int compare(File f1, File f2)
            {                 
                Integer name1Int = Integer.parseInt(f1.getName().replaceAll(".html", ""));
                Integer name2Int = Integer.parseInt(f2.getName().replaceAll(".html", ""));
                
                return name1Int.compareTo(name2Int); 
            }
        });        
        
        
        //Stopwords
        try {
            BufferedReader in= 
                    new BufferedReader(
                        new FileReader(stopwordsfile));
            String str;
            while ((str = in.readLine())!=null){
                    StringTokenizer st = new StringTokenizer(str);

                    String stopword = st.nextToken();
                    
                    this.stopWords.add(stopword);
            }
            in.close();
            
	} 
        catch (IOException e) { e.printStackTrace(); }        
        
    }

    
    public void fillWordMap() throws SQLException {
    	
    	stmt = conn.prepareStatement("insert into documents values (?,?,?)");
    	
    	System.out.println("Num of files: " + files.length);

        for (int i=0; i<this.files.length; i++) {
            
            File file = files[i];
            //System.out.println("Processing file: " + files[i].getName());
            Integer fnameInt = Integer.parseInt(file.getName().replaceAll(".html", ""));
            //fnameInt is the docid
            
            int words = 0; //num of words
            int maxfreq = 1; //max freq of any word in the document
           
            String word = null;
            try {
                FileInputStream fis = new FileInputStream(file);
                BufferedInputStream bis = new BufferedInputStream(fis);
                BufferedReader br = new BufferedReader(new InputStreamReader(bis));

                String line = null;
                while ( (line = br.readLine()) != null )  {
                    StringTokenizer st = new StringTokenizer(line, " :*_+][?!'.,;><-0123456789/()\\\"");
                    
                    while (st.hasMoreElements()) {
                        word = st.nextToken();
                        word = word.toLowerCase();
                        if(word.length() <= 2) continue;
                        if(this.stopWords.contains(word)) continue;
                        
                        SortedMap<Integer,Integer> docFreq = wordDocumentMap.get(word);
                        // word doesn't exit -> create new doc freq map
                        if(docFreq == null){
                        	docFreq = new TreeMap<Integer, Integer>();
                        	docFreq.put(fnameInt, 1);
                        	wordDocumentMap.put(word, docFreq);
                        }
                        // otherwise, increment document counter
                        else{
                        	if(docFreq.get(fnameInt) != null){
                        		docFreq.put(fnameInt, docFreq.get(fnameInt) + 1);
                        		if(docFreq.get(fnameInt) >= maxfreq)
                        			maxfreq = docFreq.get(fnameInt);
                        	}
                        	else
                        		docFreq.put(fnameInt, 1);
                        }
                        words++;
                    }
                    
                }

                insertIntoDocTable(fnameInt, words, maxfreq);

                fis.close();
                bis.close();
                br.close();
            }
            catch (Exception e) {
            	e.printStackTrace();
            }
        }        
        
        conn.close();
        stmt.close();
    } 
    
    private void insertIntoDocTable(int docid, int words, int maxfreq) throws SQLException{

		stmt.setInt(1, docid);
		stmt.setInt(2, words);
		stmt.setInt(3, maxfreq);
		stmt.executeUpdate();
    }
    
    
    //TODO fully implement
    public void insertIntoInvertedIndex() throws SQLException {
    	
    	conn = DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:TEACH", "wunan","uviccsc");
        stmt = conn.prepareStatement("insert into postings values (?,?,?)");
        
        for(String word : wordDocumentMap.keySet()){
        	
        	SortedMap<Integer,Integer> docFreq = wordDocumentMap.get(word);
        	SortedMap<Integer,String> blockMap = Encoding.Map2Strings(docFreq);
        	
        	for (Integer firstDoc : blockMap.keySet()) {
        		
				stmt.setString(1, word); //word
				stmt.setInt(2, firstDoc); //firstdoc
				stmt.setString(3, blockMap.get(firstDoc)); //block
				
				stmt.executeUpdate();
        	}
        }
        stmt.close();
        conn.close();
    }
    

    //This method is for testing only
    public void printWordDocumentMap() {
    	/*
        System.out.println( "this.wordDocumentMap.size() = " + this.wordDocumentMap.size() );
               
        for(String word : this.wordDocumentMap.keySet()) {
            System.out.print(word + ": ");
            
            String s1 = this.wordDocumentMap.get(word).toString();           
            String s2 = Utils.Strings2Map( Utils.Map2Strings(this.wordDocumentMap.get(word)).entrySet() ).toString();            

            if( !s1.equals(s2) )
                System.out.println( "ALARM: s1=" + s1 + "  s2=" +  s2);
            
            System.out.print( s2 );
            System.out.println("");
        } 
        */
    }
    
    
    public static void main(String[] args) {

        //Testing
    	try {
	        Index myIndex = new Index("html",  "stopwords.txt");
	        myIndex.fillWordMap();
	        myIndex.printWordDocumentMap();
	        
			//myIndex.insertIntoInvertedIndex();
	        
		} catch (SQLException e) {
			e.printStackTrace();
		}
    }
    
}
